# # pandas库
#
# '''
# 1.对象的创建
# '''
# # Pandas Series对象
# '''
# Series是带标签数据的一维数组
# Series对象的创建
# 通用结构：pd.seris(data,index=index,dtype=dtype)
# data:数据，可以是列表、字典、numpy数组
# index：索引，为可选参数
# dtype：数据类型，为可选参数
# '''
# # 1，用列表创建  index缺省，默认为整数序列
#
#
# import pandas as pd
#
# # data = pd.Series([1.5, 4, 3, 4.5, 6])
# # print(data)
#
# # # 增加index
# # data = pd.Series([1.5, 4, 3, 4.5], index=["a", "b", "c", "d"])
# # print(data)
#
#
# # 数据类型可被强制改变
# # data = pd.Series([1, 2, "3", 4], index=["a", "b", "c", "d"], dtype=float)
# # print(data)
#
#
# # 用一维numpy数组创建
# # import numpy as np
# # x= np.arange(5)
# # print(pd.Series(x))
#
#
# # #用字典创建 默认以键为Index,值为data
# # x= {"Beijing":2154,"Shanghai":2424,"Shenzhen":13030}
# # # print(pd.Series(x))
# #
# #
# # #字典创建，如果指定index，则会到字典的键中筛选，找不到的，值设为NaN
# # x1=pd.Series(x,index=["Beijing","Hangzhou","c"])
# # print(x1)
#
# # data为标量情况
# # a=pd.Series(5,index=[100,200,300])
# # print(a)
#
#
# '''
# Pandas DataFrame对象
# DataFrame是带标签数据的多维数组
# 通用数据结构：pd.DataFrame(data,index=index,columns=columns)
# data：数据，可以是列表、字典、或者numpy数组
# index:索引，为可选参数
# columns：列标签，为可选参数
# '''
# # 1.通过Series对象创建
# population_dict = {"Beijing": 2154, "Shanghai": 2424, "Shenzhen": 13030}
# population = pd.Series(population_dict)
# # print(pd.DataFrame(population))
# #
# #
# # print(pd.DataFrame(population, columns=["test"]))
#
# # 2.通过Series对象字典创建
# GDP_dict = {"Beijing": 30320, "Shanghai": 32680}
# GDP = pd.Series(GDP_dict)
# # print(GDP)
#
# grp = pd.DataFrame({"population": population, "DGP": GDP, "country": "China"})
# # print(grp)
#
# # 3.通过字典列表对象创建
# # 字典项作为index，字典键作为columns
# # b = [{"a": i, "b": 2 * i}
# #      for i in range(3)]
# # print(pd.DataFrame(b))
# # #不存在的键，会默认值为NaN
# # data=[{"a":1,"b":1},{"b":2,"c":4}]
# # print(pd.Series(data))
#
# # 4.通过numpy二维数组创建
# # import numpy as np
# #
# # print(pd.DataFrame(np.random.randint(10, size=(3, 2)), columns=["foo", "bar"], index=["a", "b", "c"]))
#
#
# # 5.DataFrame性质
# # 1.属性
# data = pd.DataFrame({"pop": population, "GDP": GDP})
# print(data)
#
# # df.values返回numpy数组表示的数据
# # print(data.values)
# #
# # #df.index 返回行的索引
# # print(data.index)
# #
# #
# # #df.columns返回列索引
# # print(data.columns)
# #
# # #df.shape形状
# # print(data.shape)
# #
# # #df.size大小
# # print(data.size)
# #
# # #pd.dtypes 返回每列数据类型
# # print(data.dtypes)
#
# # 2.索引
# # 2.1 获取列
# # 字典式
# # print("---")
# # print(data["pop"])
# # print("---")
# # print(data[["pop","GDP"]])
#
# #
# # print(data.GDP)
#
# # 获取行
# # 绝对索引 df.loc
# # print(data.loc("Beijing"))
# # print(data.loc[["Beijing","Shenzhen"]])
#
# # 相对索引
# # print(data.iloc[0])
# # print(data.iloc[[1,2]])
#
# # 获取标量
# # print(data.loc["Beijing","GDP"])
# # print(data.iloc[0,1])
# #
# # print(data.values[0][1])
#
# # Series对象的索引
# # print(type(data.GDP))
# # print(GDP["Beijing"])
#
# # 3.切片
# dates = pd.date_range(start='2019-01-01', periods=6)
# print(dates)
#
# import numpy as np
#
# df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=["A", "B", "C", "D"])
# print(df)
#
# # 行切片
# # print(df["2019-01-01":"2019-01-03"])
# # print(df.loc["2019-01-01":"2019-01-03"])
# # print(df.iloc[0:3])
# #
# # # 列切片
# # print(df.loc[:, "A":"C"])
# # print(df.iloc[:, 0:3])
#
#
# # 多种多样的取值
# # 行、列同时切片
# # print(df.loc["2019-01-02":"2019-01-03","C":"D"])
# # print(df.iloc[1:3,2:])
#
# # 行切片，列分散取值
# # print(df.loc["2019-01-04":"2019-01-06", ["A", "C"]])
# # print(df.iloc[3:, [0, 2]])
#
# # 行分散取值，列切片
# # print(df.loc[["2019-01-04","2019-01-06"],"C":"D"])
# # print(df.iloc[[1,5],0:3])
#
#
# # 行、列均分散取值
# # print(df.loc[["2019-01-04","2019-01-06"],["C","D"]])
# # print(df.iloc[[4,5],[0,3]])
#
#
# # 布尔索引
# # print(df>0)
# # print(df[df>0])
#
# # isin方法
# # df2= df.copy()
# # df2["E"]=["one","one","two","three","four","three"]
# # print("---")
# # print(df2)
# #
# # print("---")
# # ind=df2["E"].isin(["two","four"])
# # print(ind)
# # print("---")
# # print(df2[ind])
#
# # 赋值
# # DataFrame增加新列
# s1 = pd.Series([1, 2, 3, 4, 5, 6], index=pd.date_range(
#     '20190101', periods=6
# ))
#
# # print(s1)
# # df["E"]=s1
# # print(df)
#
#
# # 修改赋值
# df.loc["2019-01-01", "A"] = 0
# print(df)
# df.iloc[0, 1] = 2
# print(df)
#
# df["D"] = np.array([5] * len(df))  # 可简化成df["D"]=5
# print(df)
# #修改index和columns
# # df.index=[i for i in range(len(df))]
# # print(df)
# #
# # df.columns=[i for i in range(df.shape[1])]
# # print(df)
'''
2.3数值运算及统计分析
'''
# 1.数据的查看
import numpy as np
import pandas as pd

# dates = pd.date_range(start="2022-01-01",periods=6)
# df = pd.DataFrame(np.random.randn(6,4),index=dates,columns=["A","B","C","D"])
# print(df)
# 1.查看前面的行
# print(df.head()) #默认前5行
# print(df.head(2))
# 2.查看后面的行
# print(df.tail())
# print(df.tail(2))
# 3.查看总体信息
# df.iloc[0,3]=np.nan
# print(df.info)

# 2.Numpy通用函数同样适用于Pandas
# 1.向量化运算
# x=pd.DataFrame(np.arange(4).reshape(1,4))
# print(x)
# print(x+5)
# print(np.exp(x))
#
# y=pd.DataFrame(np.arange(4,8).reshape(1,4))
# print(y)
#
# print(x*y)
# 2.矩阵运算
# np.random.seed(42)
# x= pd.DataFrame(np.random.randint(10,size=(30,30)))
# # print(x)
# y= pd.DataFrame(np.random.randint(10,size=(30,30)))
# print(y)
# 求转置
# print(x.T)

# print(x.dot(y))

# 一般来说，纯粹的计算在Numpy里执行更快
# Numpy更侧重于计算，Pandas更侧重于数据处理

# 3.广播运算
# np.random.seed(42)
# x= pd.DataFrame(np.random.randint(10,size=(3,3)),columns=list("ABC"))
# print(x)

# 按行广播
print("---")


# print(x.iloc[0])
# print(x/x.iloc[0])
# 按列广播
# print(x.A)
# print(x.div(x.A,axis=0))  #add sub div mul
# print(x.div(x.iloc[0],axis=1))  #add sub div mul


# 3.新的用法
# 1.索引对齐
# A = pd.DataFrame(np.random.randint(0, 20, size=(2, 2)), columns=list("AB"))
# print(A)
# B = pd.DataFrame(np.random.randint(0, 10, size=(3, 3)), columns=list("ABC"))
# print(B)
# print(A+B)
# print(A.add(B,fill_value=0))
# pandas会自动对齐两个对象的索引，没有的值用np.nan表示
# 2.统计相关
# x=np.random.randint(3,size=20)
# print(x)
#
# print(np.unique(x))
#
# from collections import Counter
# print(Counter(x))
#
# x1=pd.DataFrame(x,columns=["A"])
# print(x1)
# print("---")
# print(np.unique(x1))
# print(x1["A"].value_counts())


# 产生新的结果，并进行排序
# population_dict={"Beijing":123,"Shanghai":456}
# population=pd.Series(population_dict)
#
# GDP_dict={"Beijing":54321,"Shanghai":43215}
# GDP=pd.Series(GDP_dict)
#
# city_info = pd.DataFrame({"population":population,"GDP":GDP})
#
# print(city_info)
#
# city_info["per_GDP"] = city_info["GDP"]/city_info["population"]
# print(city_info)
#
# #递增排序
# city_info=city_info.sort_values(by="per_GDP",ascending=True)
# # city_info.sort_values(by="per_GDP",ascending=False)
# print(city_info)


# 按轴进行排序
# data = pd.DataFrame(np.random.randint(20,size=(3,4)),index=[2,1,0],columns=list("CBAD"))
# print(data)
# #行排序
# # data=data.sort_index()
# #列排序
# data=data.sort_index(axis=1)
# print(data)


# 统计方法
# df = pd.DataFrame(np.random.normal(2, 4, size=(6, 4)), columns=list("ABCD"))
# print(df)


# 非空个数
# print(df.count())
# 求和
# print(df.sum())
# print(df.sum(axis=1))
# 最小值
# print(df.min())
# print(df.min(axis=1))

# print(df.idxmax())
# print(df.idxmin())

# 均值
# print(df.mean())
# print(df.mean(axis=1))
#
# #方差
# print(df.var())
# 众数

# print(df.mode())

# 分位数

# print(df.quantile(0.75))

# 一网打尽
# print(df.describe())
#
# #相关系数和协方差
# print(df.corr())
# print(df.corrwith(df["A"]))


# 自定义输出
# apply(method)的用法：使用method方法默认堆每一列进行相应的操作
# print(df.apply(np.cumsum))
# print(df.apply(np.cumsum,axis=1))
# print(df.apply(sum))
# print(df.apply(lambda x:x.max()-x.min()))

# def my_describe(x):
#     return pd.Series([x.count(), x.mean(), x.max(), x.idxmin(), x.std()],
#                      index=["Count", "mean", "max", "idxmin", "std"])
# print(df.apply(my_describe()))


# 缺失值处理
# 1.发现缺失值
# data=pd.DataFrame(np.array([[1,np.nan,2],
#                             [np.nan,3,4]
#                             ,[5,6,None]]),columns=["A","B","C"])
# 注意：有None、字符串等，数据类型全部变为Object，它比int和float更消耗资源
# print(data.dtypes)
# print(data.isnull())
# print(data.notnull())
# 2.删除缺失值
# data = pd.DataFrame(np.array([[1, np.nan, 2, 3], [np.nan, 4, 5, 6], [7, 8, np.nan, 9], [10, 11, 12, 13]]),
#                     columns=["A", "B", "C", "D"])
# print(data)
# 注意：np.nan是一种特殊的浮点数
# print(data.dtypes)

# 删除有nan整行
# print(data.dropna())
# print(data.dropna(axis="columns"))
# print(data.dropna(axis="columns",how="any"))
# print(data.dropna(axis="columns",how="all"))

# print(data.fillna(value=5))

# 用均值进行替换
# fill = data.stack().mean()
# print(data.fillna(value=fill))


# 合并数据
def make_df(cols, ind):
    "一个简单的DataFrame"
    data = {c: [str(c) + str(i) for i in ind] for c in cols}
    return pd.DataFrame(data, ind)


# print(make_df("ABC",range(3)))

# 垂直合并
# df_1=make_df("AB",[1,2])
# df_2=make_df("AB",[3,4])
# print(df_1)
# print(df_2)
# 垂直合并
# print(pd.concat([df_1,df_2]))
# print(pd.concat([df_1,df_2],ignore_index=True))
# 水平合并
# print(pd.concat([df_1,df_2],axis=1))


# 对齐合并 merge()
# df9=make_df("AB",[1,2])
# df10=make_df("BC",[1,2])
#
# print(pd.merge(df9,df10))


# 分组和数据透视表
# df = pd.DataFrame({"key": ["A", "B", "C", "A", "B", "C"],
#                    "data1": range(6),
#                    "data2": np.random.randint(0, 10, size=6)})
# print(df)
# 分组
# 延迟计算
# print(df.groupby("key"))
# print(df.groupby("key").sum())
# print(df.groupby("key").mean())

# for i in df.groupby("key"):
#     print(str(i))

# 按列取值
# print(df.groupby("key")["data2"].sum())

# 按组迭代
# for data,group in df.groupby("key"):
#     print("{0:5} shape={1}".format(data,group.shape))

# 支持更复杂的操作
# print(df.groupby("key").aggregate(["min","median","max"]))

# 调用方法
# print(df.groupby("key")["data1"].describe())

# 支持更复杂的操作
# 过滤
# def filter_func(x):
#     return x["data2"].std() >3
# print(df.groupby("key").filter(filter_func()))

# 转换
# print(df.groupby("key").transform(lambda x:x-x.mean()))


# 将列表、数组设为分组键
# L=[0,1,0,1,2,0]
# print(df.groupby(L).sum())

# 将字典作为分组键
# mapping={"A":"first","B":"constant","C":"constan"}
# print(df.groupby(mapping).sum())

# 任意Python函数
# print(df.groupby(str.lower()).mean())
# 多个有效值组成的列表
# print(df.groupby([str.lower(),mapping]).mean())


# 例1 行星观测数据处理
# import seaborn as sns
# planets= sns.load_dataset("planets")
# # print(planets.shape)
#
# print(planets.head())


# 例2  数据透视表  泰坦尼克号乘客数据分析
# import seaborn as sns
# titanic = sns.load_dataset("tictanic")
# print(titanic.head())


# 其他
'''
1.向量化字符串操作
2.处理时间序列
3.多级索引：用于处理多维数据
4.高性能的Pandas：eval()  query()
'''

# 3.多级索引
# base_data = np.array([[1234, 4324],
#                       [2234, 4554],
#                       [4234, 5554],
#                       [5234, 6554],
#                       [6234, 7554],
#                       [7234, 8554],
#                       [8234, 9554],
#                       [9234, 41554]])
# data = pd.DataFrame(base_data, index=[
#     ["Beijing", "Beijing","Shanghai", "Shanghai""Shenzhen", "Shenzhen""Guangzhou", "Guangzhou"]
#     , [2008, 2018] * 4],columns=["popultaion", "GDP"])
# print(data)
# 4.高性能的Pandas：eval()
# df1,df2,df3,df4=(pd.DataFrame(np.random.random((10000,100))) for i in range(4))
# %timeit (df1+df2)/(df3+df4)
# 减少了符合代数式计算中间过程的内存分配
# np.allclose((df1+df2)/(df3+df4),pd.eval("(df1+df2)/(df3+df4)"))

# eval query的使用时机
# 小数组时，普通方法反而更快
